﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Collections;
using DotNet.Common.Kit;
using System.Threading;
namespace taobaoSpider
{
    /// <summary>
    /// 采集“宝贝搜索页”数据模型。采用多线程模型设计。对于100个页面有一个总的字典，同时
    /// 对每个页面也会有一个字典，以维护每个线程的得到的掌柜名和地名，但这个字典要对照总的字典。
    /// </summary>
    public class HtmlKeySearch
    {
        private ShowErrorLogEventHandler ShowErrHandler=null;
        LogerHelper logger = new LogerHelper(DirType.Local, MessageType.Exception);
        static int ipIndex = 0, ipCount = 1; string currentIP = "本机IP";//代理服务器时用来计算取的索引和代理IP总数目
        IniHelper ini = new IniHelper("HtmlNodeXPath.ini");

        string _location = "";  //所在地，
        string _style = "grid";//显不大图还是小图(list)列表显示。
        int _s = 0;
        string _keyword = "";

        string _nickPath = ""; //输入的nick节点的XPath值。
        string _locPath = "";
        string _detailPath = "";//含有宝贝名称和宝贝详情页的节点路径。
        
        Dictionary<string, string> _nickLocSum = new Dictionary<string, string>();  //key:用户名，val:位置  100页的总表。
        Dictionary<string, string> _nickLocOnePage = new Dictionary<string, string>(); //维护每个页面的字典，要参照总字典。
        Dictionary<string,string> _nickPosition=new Dictionary<string,string>();   //页面:该面的位置。如果1:30 表示在第一页的30位。
        
        /// <summary>
        /// 得到掌柜名
        /// </summary>
        public Dictionary<string, string> NickLocationSum //维护每个页面的掌柜=>地区 字典
        {
            get { return _nickLocSum; }
            set { _nickLocSum = value; }
        }

        public Dictionary<string, string> NickLocationOnePage
        {
            get { return _nickLocOnePage; }
        }

        public Dictionary<string,string> NickPosition
        {
            get { return _nickPosition; }
        }

        public string NickXpath
        {
            set { _nickPath=value;}
        }
        public string LocalXpath
        {
            set { _locPath = value; }
        }
        public string DetailXpath
        {
            set { _detailPath = value; }
        }

        /// <summary>
        /// 传入要查询的关键词
        /// </summary>
        public string KeyWord
        {
            set { _keyword = value; }
        }
        /// <summary>
        /// 所在地
        /// </summary>
        public string Location
        {
            set { _location = value; }
        }
        /// <summary>
        /// 显示版面风格。
        /// </summary>
        public string Style
        {
            set { _style = value; }
        }
        /// <summary>
        /// 查询的页面
        /// </summary>
        public int S
        {
            set { _s = value; }
        }

        //public KeySearch()
        //{

        //}

        /// <summary>
        /// 传入写错误的委托对象
        /// </summary>
        /// <param name="showErrHandler"></param>
        //public KeySearch(ShowErrorLogEventHandler showErrHandler)
        //{
        //    this.ShowErrHandler = showErrHandler;
        //}

        /// <summary>
        /// 传入写错误的委托，Nick节点的XPath, Loc节点的XPath.必须有一个值。
        /// </summary>
        /// <param name="showErrHandler">写错误委托</param>
        /// <param name="nickXpath">NickName节点的xpath</param>
        /// <param name="locXpath">Loc节点的Xpath</param>
        public HtmlKeySearch(ShowErrorLogEventHandler showErrHandler, string nickXpath, string locXpath,string detailXpath)
        {
            this.ShowErrHandler = showErrHandler;
            this._nickPath = nickXpath;
            this._locPath = locXpath;
            this._detailPath = detailXpath;
        }


        private string SetRequestUrl()
        {
            string url = "";
            if (_location == "")
            {
                url = "http://s.taobao.com/search?q=" + System.Web.HttpUtility.UrlEncode(_keyword) +
                    "&s=" + _s*52 + "&style=" + _style;
            }
            else
            {
                url = url = "http://s.taobao.com/search?q=" + System.Web.HttpUtility.UrlEncode(_keyword) +
                    "&s=" + _s*52 + "&style=" + _style+"&loc="+System.Web.HttpUtility.UrlEncode(_location);
            }
            return url;
        }
        /// <summary>
        /// 初始化关键词搜索页的数据，按页面数量划分线程数，默认为100个线程。
        /// </summary>
        /// <returns></returns>
        //public int InitNickInfo(int threadID, string threadName)
        //{
        //    _s = threadID;
        //    HttpHelper httpHelper = new HttpHelper();
        //    HttpItem item = new HttpItem();
        //    item.URL = SetRequestUrl();
        //    string htmlStr = httpHelper.GetHtml(item).Html;
        //    //GlobalFun.Write("d:\\" + i.ToString() + ".html", true, htmlStr);
        //    //htmlStr.Contains("操作超时") || htmlStr.Contains("读取异常") || htmlStr.Contains("无法连接到远程服务器") || htmlStr.Contains("请求被中止") || htmlStr.Contains("操作已超时") || htmlStr.Contains("远程服务器返回错误") || htmlStr.Contains("访问受限了") || htmlStr.Contains("Please wait while you are redirected ...")
        //    while (!htmlStr.Contains("淘宝搜索"))
        //    {
        //        GlobalFun.WriteErrorLog("KeySearch:InitNickInfo():while", "htmlStr");
        //        if (htmlStr.Contains("无法连接到远程服务器") || htmlStr.Contains("远程服务器返回错误"))
        //        {
        //            Router.DisConnect();
        //        }
        //        System.Threading.Thread.Sleep(100);
        //        htmlStr = httpHelper.GetHtml(item).Html;
        //    }
        //    HtmlAgilityPack.HtmlDocument doc = GlobalFun.ParaseToHtml(htmlStr);
        //    HtmlAgilityPack.HtmlNodeCollection nickHnc = doc.DocumentNode.SelectNodes(_nickPath);
        //    HtmlAgilityPack.HtmlNodeCollection locHnc = doc.DocumentNode.SelectNodes(_locPath);
        //    if (nickHnc != null && locHnc != null)
        //    {
        //        _nickLocOnePage.Clear();
        //        for (int index = 0; index < nickHnc.Count; index++)
        //        {
        //            string keyNickName=nickHnc[index].InnerText.Trim();
        //            string valueLoc=locHnc[index].InnerText.Trim();
        //            if (!_nickLocSum.ContainsKey(keyNickName))
        //            {
        //                _nickLocSum[keyNickName] = valueLoc;
        //                _nickLocOnePage[keyNickName] = valueLoc;
        //            }   
        //        }
        //    }
        //    foreach (KeyValuePair<string, string> kvp in _nickLocSum)
        //    {
        //        GlobalFun.Write("d:\\taobao1\\" + "sum" +threadName+ ".txt", true, kvp.Key + ":" + kvp.Value);
        //    }
        //    // return 0;
        //    return 0;
        //}
        /// <summary>
        /// 解析While循环中的htmlStr得到相应的值。
        /// </summary>
        /// <param name="htmlStr"></param>
        /// <returns></returns>
        private string ParseHtmlStrWhile(string htmlStr)
        {
            if (htmlStr.Contains("title"))  //说明返回为一个HTML页面
            {
                HtmlAgilityPack.HtmlDocument doc = GlobalFun.ParaseToHtml(htmlStr);
                htmlStr = doc.DocumentNode.SelectNodes("//title")[0].InnerText.Trim('\r', '\n');
                return htmlStr;
            }
            else
            {
                if (htmlStr.Contains("body"))
                {
                    HtmlAgilityPack.HtmlDocument doc = GlobalFun.ParaseToHtml(htmlStr);
                    htmlStr = doc.DocumentNode.SelectNodes("//body")[0].InnerText.Trim('\r', '\n');
                    return htmlStr;
                }
                else
                {
                    return htmlStr;
                }
            } 
        }
        /// <summary>
        /// 根据关键词和掌柜字典加入本次不重复的掌柜名。
        /// </summary>
        /// <param name="keyword">关键词</param>
        /// <param name="nickSumDict">本次采集关键中的掌柜字典，保证不重复</param>
        /// <returns></returns>
        public int InitNickInfo(string keyword,Dictionary<string,string>nickSumDict)
        {
            DateTime dt1 = DateTime.Now;
            this._keyword = keyword;
            HttpHelper httpHelper = new HttpHelper();
            HttpItem item = new HttpItem();
            item.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.0; .NET CLR 1.1.4322; .NET CLR 2.0.50215;)";
            ShowErrorLogEventArgs e = new ShowErrorLogEventArgs();
            System.Windows.Forms.Label lblInfo = new System.Windows.Forms.Label();
            lblInfo.Name = "lblInfo";
            System.Windows.Forms.Label lblIpInfo = new System.Windows.Forms.Label();
            lblIpInfo.Name = "lblIpInfo";

            for (int i = 0; i < 1; i++)
            {
                _s = i;
                item.URL = SetRequestUrl();
                string htmlStr = httpHelper.GetHtml(item).Html;  //while循环变量初始值
                e.ErrPosition="KeySearch:InitNickInfo(p1,p2):for："+(_s+1)+ "页";
                e.ErrMessage = "本次使用IP：" + currentIP;
                this.ShowErrHandler(lblIpInfo, e);
                DateTime closeInitTime = DateTime.Now; //标识关机时间起点。
                while (!htmlStr.Contains("淘宝搜索"))//应该有一个退出机制，避免出现死循环
                {
                    DateTime closeEndTime = DateTime.Now;
                    string htmlStrTitle = ParseHtmlStrWhile(htmlStr); //用来标识进入路由重启或换IP的标识串。
                    e.ErrPosition = "KeySearch:InitNickInfo(string keyword,Dictionary<string,string>nickSumDict):while";
                    //////start  更新数据要联动
                    logger.LogToFile(e.ErrPosition + e.ErrMessage);
                    e.ErrMessage = htmlStrTitle + "  采集关键词:" + keyword;
                    this.ShowErrHandler(lblInfo, e);
                    ///////////end

                    if (closeEndTime > closeInitTime.AddMinutes(5)) //超时10分钟则进入关机程序或者换代理IP
                    {
                        if (ini.GetKeyValue("ProxyUseFlag", "closeSys") == "0")  //进入关机程序
                        {
                            e.ErrMessage = "开始循环时间：" + closeInitTime + "  结束循环关机时间" + closeEndTime;
                            logger.LogToFile(e.ErrPosition + "--超时5分钟关机--" + htmlStrTitle);
                            e.ErrMessage = "--超时5分钟关机--" + htmlStrTitle;
                            ShowErrHandler(lblInfo, e);
                            e.ErrMessage = "可用代理IP数：-1" + "\r\n" + "IP索引：-1" + "\r\n" + "本次使用IP：" + currentIP;
                            this.ShowErrHandler(lblIpInfo, e);

                            this.ShowErrHandler("closeSys", e);//调用关机程序
                        }
                        if (ini.GetKeyValue("ProxyUseFlag", "closeSys") == "1")  //换是个机IP访问
                        {
                            if (currentIP == "本机IP" || currentIP == "")
                            {
                                item.ProxyIp = "";
                                TBWS.tb_wsSoapClient client = new TBWS.tb_wsSoapClient();
                                currentIP = client.GetReqIp() + "(本机IP)";
                            }
                            htmlStr = httpHelper.GetHtml(item).Html;
                            htmlStrTitle = ParseHtmlStrWhile(htmlStr); //用来标识进入路由重启或换IP的标识串。
                            //以下三者信息进行联动，便于监测程序运行状况
                            logger.LogToFile(e.ErrPosition + "--超时5分钟换本机IP--" + htmlStrTitle);
                            e.ErrMessage = "--超时5分钟换本机IP--" + htmlStrTitle;
                            this.ShowErrHandler(lblInfo, e);
                            e.ErrMessage = "可用代理IP数：-1" + "\r\n" + "IP索引：-1" + "\r\n" + "本次使用IP：" + currentIP;
                            this.ShowErrHandler(lblIpInfo, e);
                        }
                    }
                    //读取INI中重启路由的条件
                    Dictionary<string, string> dict = ini.SectionValuesEx("ReConnectRouter"); //读取INI中设定的重启路由的条件
                    if (GlobalFun.ExistDict(dict, htmlStrTitle))  //换IP的流程，此处分两部分，一是重启路由(0)，一个是使用代理(1)。
                    {
                        if (ini.GetKeyValue("ProxyUseFlag", "proxy") == "0")
                        {
                            currentIP = Router.DisConnect();
                            logger.LogToFile(e.ErrPosition + "重启路由IP:" + currentIP);
                            e.ErrMessage = htmlStrTitle + "  关键词:" + this._keyword;
                            ShowErrHandler(lblInfo, e);
                            e.ErrMessage = "重启路由IP--" + ipCount + "\r\n" + "IP索引--" + ipIndex + "\r\n" + "本次使用IP--" + currentIP;
                            this.ShowErrHandler(lblIpInfo, e);
                        }
                        if (ini.GetKeyValue("ProxyUseFlag", "proxy") == "1")
                        {
                            //if (ipCount != 0)
                            //    ipIndex = ipIndex % ipCount;
                            ipIndex = 0;
                            Router.ShowErrHandler = this.ShowErrHandler;
                            string vtime = "";
                            currentIP = Router.ProxyIp(ref ipIndex, ref ipCount,ref vtime);
                            if (currentIP == "本机IP" || currentIP == "")
                            {
                                item.ProxyIp = "";
                                TBWS.tb_wsSoapClient client = new TBWS.tb_wsSoapClient();
                                currentIP = client.GetReqIp() + "(本机IP)";
                            }
                            logger.LogToFile(e.ErrPosition + "代理IP数--" + ipCount + "--第" + ipIndex + "--号IP：" + currentIP);
                            e.ErrMessage = htmlStrTitle+"  关键词:"+this._keyword;
                            ShowErrHandler(lblInfo, e);
                            e.ErrMessage = "可用代理IP数：" + ipCount + "\r\n" + "IP索引：" + ipIndex + "\r\n" + "本次使用IP：" + currentIP + "(速度:" + vtime + " s)";
                            this.ShowErrHandler(lblIpInfo, e);
                            //ipIndex++;
                        }
                    }
                    htmlStr = httpHelper.GetHtml(item).Html; 
                }
                if (htmlStr.Contains("筛选条件加的太多啦"))
                {
                    break;
                }
                HtmlAgilityPack.HtmlDocument doc = GlobalFun.ParaseToHtml(htmlStr);
                HtmlAgilityPack.HtmlNodeCollection nickHnc = doc.DocumentNode.SelectNodes(_nickPath);
                HtmlAgilityPack.HtmlNodeCollection locHnc = doc.DocumentNode.SelectNodes(_locPath);
                HtmlAgilityPack.HtmlNodeCollection detailHnc = doc.DocumentNode.SelectNodes(_detailPath);
                if (nickHnc != null && locHnc != null)
                {
                    _nickLocOnePage.Clear(); //对每一页的掌柜信息加入字典
                    for (int index = 0; index < nickHnc.Count; index++)
                    {
                        string keyNickName = nickHnc[index].InnerText.Trim();
                        string valueLoc = locHnc[index].InnerText.Trim();
                        string detailUrl = detailHnc[index].Attributes["href"].Value; //宝贝详情URL
                        string bbName = detailHnc[index].Attributes["title"].Value.Trim();  //宝贝名称
                        if (!nickSumDict.ContainsKey(keyNickName))
                        {
                            _nickLocSum[keyNickName] = valueLoc + "*" + bbName + "*" + detailUrl;
                            _nickLocOnePage[keyNickName] = valueLoc;
                            nickSumDict[keyNickName] = valueLoc + "*" + bbName + "*" + detailUrl;
                        }
                    }
                }

                e.ErrMessage = (DateTime.Now-dt1).TotalMinutes.ToString();
                System.Windows.Forms.Label lblUseTime = new System.Windows.Forms.Label();
                lblUseTime.Name = "lblUseTime";
                this.ShowErrHandler(lblUseTime, e);

                e.ErrPosition = "KeySearch:InitNickInfo()=>for";
                e.ErrMessage = "正在采集关键词<<" + this._keyword + ">>的第 " + (i + 1).ToString() + " 页 共有不重复掌柜信息 " + _nickLocOnePage.Count + " 条";
                this.ShowErrHandler(lblInfo, e);
                //e.ErrMessage = "可用代理IP数--" + ipCount + "\r\n" + "IP索引--" + ipIndex + "\r\n" + "本次使用IP--" + currentIP;
                //this.ShowErrHandler(lblIpInfo, e);

                //e.ErrPosition = "";
                e.ErrMessage = nickSumDict.Count.ToString();
                System.Windows.Forms.Label lblSum = new System.Windows.Forms.Label();
                lblSum.Name = "lblSum";
                this.ShowErrHandler(lblSum, e);

                e.ErrPosition = this._keyword;
                e.ErrMessage = "0:" + this._nickLocSum.Count.ToString();
                System.Windows.Forms.DataGridView dgvKeyword = new System.Windows.Forms.DataGridView();
                dgvKeyword.Name = "dgvKeyword";
                this.ShowErrHandler(dgvKeyword, e);
            }
            return 0;
        }
        /// <summary>
        /// 
        /// </summary>
        /// <returns></returns>
        //public int InitNickInfo()
        //{
        //    HttpHelper httpHelper = new HttpHelper();
        //    HttpItem item = new HttpItem();
        //    ShowErrorLogEventArgs e = new ShowErrorLogEventArgs();
        //    System.Windows.Forms.Label lblInfo = new System.Windows.Forms.Label();
        //    lblInfo.Name = "lblInfo";
        //    for (int i = 0; i < 2; i++)
        //    {
        //        _s = i;
        //        item.URL = SetRequestUrl();
        //        string htmlStr = httpHelper.GetHtml(item).Html;
        //        DateTime closeInitTime = DateTime.Now; //标识关机时间起点。
        //        while (!htmlStr.Contains("淘宝搜索"))//超时30分钟则进入关机程序
        //        {
        //            DateTime closeEndTime = DateTime.Now;
        //            if (closeEndTime > closeInitTime.AddMinutes(30))  //超时30分钟则进入关机程序
        //            {
        //                e.ErrMessage = "开始循环时间：" + closeInitTime + "  结束循环关机时间" + closeEndTime;
        //                logger.LogToFile("KeySearch:InitNickInfo():while---"+e.ErrMessage);
        //                frmCloseSystem closeFrm = new frmCloseSystem();
        //                closeFrm.ShowDialog();
        //            }

        //            GlobalFun.WriteErrorLog("KeySearch:InitNickInfo():while",ParseHtmlStrWhile(htmlStr));
        //            if (htmlStr.Contains("无法连接到远程服务器") || htmlStr.Contains("远程服务器返回错误") || htmlStr.Contains("您访问的页面不存在") || htmlStr.Contains("天猫Tmall.com-上天猫"))
        //            {
        //                Router.DisConnect();
        //                System.Threading.Thread.Sleep(3000);
        //            }
        //            htmlStr = httpHelper.GetHtml(item).Html;
        //        }
        //        if (htmlStr.Contains("筛选条件加的太多啦"))
        //        {
        //            break;
        //        }
        //        HtmlAgilityPack.HtmlDocument doc = GlobalFun.ParaseToHtml(htmlStr);
        //        HtmlAgilityPack.HtmlNodeCollection nickHnc = doc.DocumentNode.SelectNodes(_nickPath);
        //        HtmlAgilityPack.HtmlNodeCollection locHnc = doc.DocumentNode.SelectNodes(_locPath);
        //        if (nickHnc != null && locHnc != null)
        //        {
        //            _nickLocOnePage.Clear();
        //            for (int index = 0; index < nickHnc.Count; index++)
        //            {
        //                string keyNickName = nickHnc[index].InnerText.Trim();
        //                string valueLoc = locHnc[index].InnerText.Trim();
        //                if (!_nickLocSum.ContainsKey(keyNickName))
        //                {
        //                    _nickLocSum[keyNickName] = valueLoc;
        //                    _nickLocOnePage[keyNickName] = valueLoc;
        //                }
        //            }
        //        }
        //    }
        //    return 0;
        //}
    }
}